{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST Outlier Detection (WIP)\n",
    "\n",
    "Skaffolding for MNIST Outlier Detection demo.\n",
    "\n",
    "Create Minikube:\n",
    "\n",
    "```\n",
    "minikube start --vm-driver kvm2 --memory 4096 --feature-gates=CustomResourceValidation=true --extra-config=apiserver.Authorization.Mode=RBAC\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import requests\n",
    "from random import randint,random\n",
    "import json\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import sys\n",
    "sys.path.append(\"../../../notebooks\")\n",
    "from visualizer import get_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_image(arr):\n",
    "    two_d = (np.reshape(arr, (28, 28)) * 255).astype(np.uint8)\n",
    "    plt.imshow(two_d,cmap=plt.cm.gray_r, interpolation='nearest')\n",
    "    return plt\n",
    "\n",
    "def download_mnist():\n",
    "    return input_data.read_data_sets(\"MNIST_data/\", one_hot = True)\n",
    "\n",
    "def rest_predict_request(endpoint,data):\n",
    "    request = {\"data\":{\"ndarray\":data.tolist()}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/predict\",\n",
    "                data={\"json\":json.dumps(request),\"isDefault\":True})\n",
    "    return response.json()   \n",
    "\n",
    "def rest_transform_input_request(endpoint,data):\n",
    "    request = {\"data\":{\"ndarray\":data.tolist()}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/transform-input\",\n",
    "                data={\"json\":json.dumps(request),\"isDefault\":True})\n",
    "    return response.json()   \n",
    "\n",
    "def rest_transform_output_request(endpoint,data):\n",
    "    request = {\"data\":{\"ndarray\":data.tolist()}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/transform-output\",\n",
    "                data={\"json\":json.dumps(request),\"isDefault\":True})\n",
    "    return response.json()   \n",
    "\n",
    "def rest_request_ambassador(deploymentName,endpoint=\"localhost:8003\",arr=None):\n",
    "    payload = {\"data\":{\"names\":[\"a\",\"b\"],\"tensor\":{\"shape\":[1,784],\"values\":arr.tolist()}}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/seldon/\"+deploymentName+\"/api/v0.1/predictions\",\n",
    "                json=payload)\n",
    "    print(response.status_code)\n",
    "    print(response.text)\n",
    "\n",
    "\n",
    "def gen_mnist_data(mnist):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(1)\n",
    "    chosen=0\n",
    "    gen_image(batch_xs[chosen]).show()\n",
    "    data = batch_xs[chosen].reshape((1,784))\n",
    "    return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = download_mnist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl create namespace seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl config set-context $(kubectl config current-context) --namespace=seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl -n kube-system create sa tiller\n",
    "!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n",
    "!helm init --service-account tiller"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl rollout status deploy/tiller-deploy -n kube-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-crd --name seldon-core-crd \\\n",
    "    --set usage_metrics.enabled=true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!helm install ../../../helm-charts/seldon-core --name seldon-core \\\n",
    "        --namespace seldon \\\n",
    "        --set ambassador.enabled=true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl rollout status deploy/seldon-core-seldon-cluster-manager -n seldon\n",
    "!kubectl rollout status deploy/seldon-core-seldon-apiserver -n seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!helm install ../../../helm-charts/seldon-single-model \\\n",
    "    --set outlier_detection.enabled=true \\\n",
    "    --set outlier_detection.image.name=seldonio/outlier_mahalanobis:0.3 \\\n",
    "    --set outlier_detection.n_components=3 \\\n",
    "    --name seldon-model --set oauth.key=oauth-key \\\n",
    "    --set oauth.secret=oauth-secret \\\n",
    "    --set model.image.name=seldonio/sk-mnist:0.1 \\\n",
    "    --namespace=seldon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Port forward Ambassador**\n",
    "\n",
    "```\n",
    "kubectl port-forward $(kubectl get pods -n seldon -l service=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test with Real Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = gen_mnist_data(mnist)\n",
    "data = data.reshape((784))\n",
    "rest_request_ambassador(\"seldon-model\",endpoint=\"localhost:8003\",arr=data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test with Random Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def create_random_data(data_size,rows=1):\n",
    "    shape = [rows,data_size]\n",
    "    arr = np.random.rand(rows*data_size)\n",
    "    return (shape,arr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "shape,data = create_random_data(784)\n",
    "rest_request_ambassador(\"seldon-model\",endpoint=\"localhost:8003\",arr=data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analytics and Load Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n",
    "    --set grafana_prom_admin_password=password \\\n",
    "    --set persistence.enabled=false \\\n",
    "    --namespace seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') role=locust"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!helm install seldon-core-loadtesting --name loadtest  \\\n",
    "    --namespace seldon \\\n",
    "    --repo https://storage.googleapis.com/seldon-charts \\\n",
    "    --set locust.script=mnist_rest_locust.py \\\n",
    "    --set locust.host=http://test-deployment:8000 \\\n",
    "    --set oauth.enabled=false \\\n",
    "    --set locust.hatchRate=1 \\\n",
    "    --set locust.clients=1 \\\n",
    "    --set loadtest.sendFeedback=1 \\\n",
    "    --set locust.minWait=0 \\\n",
    "    --set locust.maxWait=0 \\\n",
    "    --set replicaCount=1 \\\n",
    "    --set data.size=784\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You should port-foward the grafana dashboard\n",
    "\n",
    "```\n",
    "kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3000:3000\n",
    "```\n",
    "\n",
    "You can then view an analytics dashboard inside the cluster at http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1. Your IP address may be different. get it via minikube ip. Login with:\n",
    "\n",
    "    Username : admin\n",
    "\n",
    "    password : password (as set when starting seldon-core-analytics above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
